﻿import wExceptions
from threading import Thread

crawlerDelay = 300
crawlerList = []
initialized = False
def init(db):
	global initialized, database, crawler, crawlerList, crawlerDelay
	database = db
	crawler = __Crawler()
	initialized = True
	
def fetch():
	"""Fetch a single item"""
	if (initialized):
		pass
	else:
		raise wExceptions.InitError()
		
def fetchAll():
	"""Fetch all current items, returns a generator"""
	if (initialized):
		pass
	else:
		raise wExceptions.InitError()

def status():
	"""Returns the status of the crawler, can be any of the following:
	
	
	"""
	pass
def add(name):
	global crawlerList
	if (initialized):
		try:
			mod = __import__('crawlers.' + name)
			if (hasattr(mod, 'crawl')):
				crawlerList[name] = mod.crawl
			else:
				raise wExceptions.CrawlerInvalidError(name, 'No crawl method')
		except (ImportError):
			raise wExceptions.CrawlerLoadError(name)
	else:
		raise wExceptions.InitError()
		
def remove(name):
	global crawlerList
	if (initialized):
		if (name in crawlerList):
			del crawlerList[name]
		else:
			raise wExceptions.CrawlerNotLoadedError(name)
	else:
		raise wExceptions.InitError()
		
class __Crawler(Thread):
	def __init__(self):
		Thread.__init__(self)
		self.running = True
		self.active = True
		self.start()
	def run(self):
		while (self.running):
			while (self.active):
				self.__status = 1
				for crawl in crawlerList.items():
					self.currentCrawler = crawl[0]
					try:
						crawl[1](self.__receiver, self.__currentCrawlerProgress)
					except:
						pass
				self.__status = 0
				time.sleep(crawlerDelay)
			time.sleep(0.5)
			
	def __currentCrawlerProgress(self, percent, info=None):
		"""Method that can be called with the overall progress
		of the crawling and any extra info as second argument
		
			'percent' has to be castable to a float
			
			'info' has to be unicode encodeable
		"""
		try:
			self.CrawlerStatus['Progress'] = float(percent)
		except (ValueError):
			raise TypeError("currentCrawlerProgress argument not float ({value} given)".format(value=percent))
		try:
			self.CrawlerStatus['Info'] = unicode(info, 'utf-8', 'replace')
		except (UnicodeDecodeError):
			raise TypeError("currentCrawlerProgress argument not unicode")
	
	def __receiver(self, data):
		"""Method that should be called on every new item found
		while crawling.
		
			'data' must be a dictionary like object with the
					assigned data key values (see ../docs/data.txt).
					
					NOTE that adding your own key values is 
					accepted but they won't be used by the
					rest of the application.